# -*- coding: utf-8 -*-
from crawler.items import *
from crawler.core.service_owner import *
from crawler.core.dict_url_city import *
from crawler.core.dict_xpath import *
import scrapy, re, logging, json, os
from crawler import settings

class RentByOwner(scrapy.Spider):
    name = "chuzu.owner.58.list"
    custom_settings = settings.MIDDLEWARES_RENT_FIRST

    def __init__(self, urls_tag='chuzu_owner_58_sd_jn', *args, **kwargs):
        super(RentByOwner, self).__init__(*args, **kwargs)
        self.rent_service = RentOwnerService(urls_tag=urls_tag)

    def start_requests(self):
        start_urls = self.rent_service.get_crawler_urls()
        for url in start_urls:
            meta = {
                'download_timeout': self.custom_settings['DOWNLOAD_TIME'],
                'referer':url.split('0/')[0]
            }
            yield scrapy.Request(url, dont_filter=True, meta=meta, callback=self.list_parse)

    def list_parse(self, response):
        if 'callback.58.com' in response.url:
            redirect_urls = response.meta.get('redirect_urls', '')
            logging.info('\n--->\n url: {0}\n NEED VERIFY \n<---'.format(redirect_urls))
        else:

            # 获取爬去城市的中文名字
            base_url = re.search(r'\w+\.58\.com', response.url).group()
            city_belong = URL_TO_CITY.get(base_url, '')
            if not city_belong: raise Exception('Not Found City Belong')

            response_ = self.rent_service.replace_font(response)
            house_items = response_.xpath(RENT_OWNER_58['items'])
            for it in house_items:
                house_info = RentOfOwner()
                try:
                    house_id = it.xpath(RENT_OWNER_58['item_id'])[0]
                    title = it.xpath(RENT_OWNER_58['title'])[0]
                    baseinfo = it.xpath(RENT_OWNER_58['baseinfo'])[0]
                    community_name1 = it.xpath(RENT_OWNER_58['community_name1'])
                    community_name2 = it.xpath(RENT_OWNER_58['community_name2'])
                    unit_price = it.xpath(RENT_OWNER_58['unit_price'])[0]
                    published_time = it.xpath(RENT_OWNER_58['published_time'])[0]
                    detail_link = it.xpath(RENT_OWNER_58['detail_link'])[0]

                    house_info['city_belong'], house_info['web_belong'] = city_belong, RENT_OWNER_58['web']
                    house_info['house_id'] = re.match(r'\w?_\d?_\d+_(\d+)_\d?_\d?_sortid', house_id).group(1)
                    house_info['title'] = title.replace('\xa0', '').replace('\n','').strip(' ')
                    house_info['lease_type'] = '整租' if '整租' in title else '合租'
                    for i in re.findall(r'(\d+)室(\d+)厅(\d+)卫|(\d+)室(\d+)厅', baseinfo):
                        house_info['room'] = i[0] if i[0] else i[3]
                        house_info['hall'] = i[1] if i[1] else i[4]
                        house_info['toilet'] = i[2] if i[2] else ''
                    house_info['lease_room'] = '主卧' if '主卧' in baseinfo else '次卧' if '次卧' in baseinfo else ''
                    for i in re.findall(r'(\d+\.\d+)㎡|(\d+)㎡', baseinfo):
                        house_info['builtuparea'] = i[0] if i[0] else i[1]
                    house_info['community_name'] = community_name1[0].replace(' ','').replace('\n','') if len(community_name1)>0 else community_name2[0].replace(' ','').replace('\n','')
                    house_info['unit_price'] = unit_price
                    house_info['published_time'] = re.search(r'postdate:(\d+)', published_time).group(1) if published_time else ''
                    house_info['detail_url'] = '%s%s' % ('https:', detail_link) if not detail_link.startswith('http') else detail_link
                    house_info['created_time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                    if not house_info['title'] or not house_info['lease_type'] or not house_info['unit_price']:
                        logging.exception('\n*** ***\n Info Not Complete\n*** ***')
                    yield house_info
                except Exception as e:
                    logging.exception('\n*** ***\n chuzu 58 owner \n {0}\n Exception {1}\n*** ***'.format(house_info['house_id'], str(e)))